﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using DataProcessor.Core;
using LumenWorks.Framework.IO.Csv;

namespace DataProcessor.FashionData
{
    public class FashionDataProcessor
    {
        public FashionDataProcessor()
        { 
        
        }

        /// <summary>
        /// Create a csv file for the input of AMT HIT
        /// </summary>
        /// <param name="inputFile">Csv file of images and categories</param>
        /// <param name="wikiFile">Csv file of wiki definitions</param>
        /// <param name="outputFile">Output file to save result</param>
        public void CreateHitInputFiles(string inputFile, string wikiFile, string outputFile)
        {
            int recordNo = 1;
            int anchor = 4;
            char sep = ',';

            // Read wiki definitions from wiki definitions files
            var wikiDefs = File.ReadAllLines(wikiFile).ToCsvDictionary('|')
                .Select(i => new
                {
                    Category = i["Category"],
                    SampleImage = i["SampleImage"],
                    Wiki = String.Format("\"{0}\"", i["WikiDef"])
                });

            // 1) Read original images, 
            // 2) Group based on categories,
            // 3) Put each 4 image in 1 record
            // 4) Join with wiki definition file for each category
            // 5) Output to csv
            var output = File.ReadAllLines(inputFile).ToCsvDictionary()
                .Select(i => new { Category = i["Category"], Url = i["Url"] })
                .GroupBy(i => i.Category).Select(g =>
                {
                    int count = 0;
                    return new
                    {
                        Category = g.Key,
                        Urls = g.Aggregate("",
                            // Concatenate urls of each group with format (with ANCHOR 3): "ur1,url2,url3 url4,url5,url6"
                            (res, i) => res += i.Url + ((count++ % anchor == 0) ? " " : sep.ToString()),
                            res => res.Substring(0, res.Length - 1)).Split(' ')
                    };
                })
                .SelectMany(c => c.Urls.Select(u => new { Category = c.Category, Urls = u }))
                .Join(wikiDefs, i => i.Category, w => w.Category, (i, w) =>
                {
                    int urlCount = i.Urls.Split(sep).Length;

                    // Generate Url css classes based on number of urls in each record. 
                    // e.g. with urlCount = 2 : "visible,visible,invisible,invisible"
                    string urlClasses = Enumerable.Range(0, anchor)
                        .Select(j => (j < urlCount) ? "visible" : "invisible")
                        .Aggregate((res, next) => res + sep + next);

                    string template = String.Join(sep.ToString(), "{0},{1},{2},{3},{4},{5},{6}".Split(','));
                    return String.Format(template,
                        recordNo++,
                        i.Category,
                        w.Wiki,
                        w.SampleImage,
                        urlCount,
                        urlClasses,
                        i.Urls
                    );
                });


            string[] fields = "ID,Category,WikiDefinition,SampleImage,UrlCount,class1,class2,class3,class4,Url1,Url2,Url3,Url4".Split(',');
            var header = new String[] { String.Join(sep.ToString(), fields) };

            File.WriteAllLines(outputFile, header.Concat(output));
        }


        /// <summary>
        /// Apply rejection to the result of AMT Hits
        /// </summary>
        /// <param name="rejectFile">A csv file specifiying which Assignments should be rejectedd</param>
        /// <param name="originalFile">Original result file downloaded from AMT</param>
        /// <param name="outputFile">Result of applying rejection which can be uploaded to AMT</param>
        public void ApplyRejection(string rejectFile, string originalFile, string outputFile)
        {
            string reject1 = "The assignments were automatically filtered and then manually and independently reviewed by two experts. There is at least one obligatory question that is not answered in your assignment.";
            string reject2 = "The Assignments were automatically filtered and then manually and independently reviewed by two experts. Both of reviewers agreed that there is mismatch between the answers of the first and second question for one or more images.";

            var rejectList = File.ReadAllLines(rejectFile).Skip(1).Select(line =>
            {
                string[] parts = line.Split(',');
                return new { AssignmentId = parts[0], RejectReason = parts[1] };
            });

            var processedLines = File.ReadAllLines(originalFile).Skip(1).ToList().Select(line =>
            {
                string reject = rejectList.Where(item => line.Contains(item.AssignmentId)).Select(item => { return item.RejectReason; }).FirstOrDefault();

                if (reject == "1")
                    return String.Format("{0},\"\",\"{1}\"", line, reject1);
                else if (reject == "2")
                    return String.Format("{0},\"\",\"{1}\"", line, reject2);
                else
                    return String.Format("{0},\"x\",\"\"", line);
            });

            File.WriteAllLines(outputFile, processedLines);
        }

        
        /// <summary>
        /// Get the answers of turkers from each HIT and save them as per image csv file
        /// </summary>
        /// <param name="inputFile">Original AMT result file</param>
        /// <param name="outputFile"></param>
        public void GetAnswersPerImage(string inputFile, string outputFile)
        {
            var fields = File.ReadAllLines(inputFile).Take(1).FirstOrDefault().Split('|')
                .Select((item, index) => new { Index = index, Name = item }).ToDictionary(item => item.Name, item => item.Index);

            var content = File.ReadAllLines(inputFile).Skip(1)
                .Select(line => new Assignment(line.Split('|'), fields))
                .SelectMany(a => Enumerable.Range(1, a.UrlCount).Select(i => new ImageInfo(a, i)))
                //.Where(ii => ii.Approved)
                .GroupBy(ii => ii.Url)
                .Where(ii => ii.Count() == 3)
                .Select(g => new ImageInfo
                {
                    Url = g.Key,
                    Related = g.Aggregate(" ", (current, i) => current != " " ? current + "," + i.Related : i.Related),
                    Specialty = g.Aggregate(" ", (current, i) => current != " " ? current + "," + i.Specialty : i.Specialty),
                    Familiarity = g.Aggregate(" ", (current, i) => current != " " ? current + "," + i.Familiarity : i.Familiarity),
                    Category = g.Select(i => i.Category).FirstOrDefault()
                }.ToString());

            List<string> output = new List<string>() { "PictureURL,Category,T1_Q1,T2_Q1,T3_Q1,T1_Q2,T1_Q2,T1_Q2,T1_Familiarity,T2_Familiarity,T3_Familiarity" };
            File.WriteAllLines(outputFile, output.Concat(content));
        }

        /// <summary>
        /// Get statistic of turkers answers for each image
        /// </summary>
        /// <param name="inputFile">Oringinal AMT result file</param>
        /// <param name="outputFile"></param>
        public void GetAnswersStatisticsPerImage(string inputFile, string outputFile)
        {
            var fields = File.ReadAllLines(inputFile).Take(1).FirstOrDefault().Split('|')
                .Select((item, index) => new { Index = index, Name = item }).ToDictionary(item => item.Name, item => item.Index);

            var content = File.ReadAllLines(inputFile).Skip(1)
                .Select(line => new Assignment(line.Split('|'), fields))
                .SelectMany(a => Enumerable.Range(1, a.UrlCount).Select(i => new ImageInfo(a, i)))
                //.Where(ii => ii.Approved)
                .GroupBy(ii => ii.Url)
                .Where(ii => ii.Count() == 3)
                .Select(g => String.Format("{0},{1},{2},{3},{4},{5},{6},{7}",
                    g.Key,
                    g.Select(i => i.Category).FirstOrDefault(),
                    g.Where(i => i.Related == "Yes").Count(),
                    g.Where(i => i.Related == "No").Count(),
                    g.Where(i => i.Related == "NotSure").Count(),
                    g.Where(i => i.Specialty == "Yes").Count(),
                    g.Where(i => i.Specialty == "No").Count(),
                    g.Where(i => i.Specialty == "NotSure").Count()
                ));

            List<string> output = new List<string>() { "PictureURL,Category,Q1CountYes,Q1CountNo,Q1CountNotsure,Q2CountYes,Q2CountNo,Q2CountNotsure" };
            File.WriteAllLines(outputFile, output.Concat(content));
        }

        /// <summary>
        /// Filter non-experts annotations and get only those which are annotated also by experts
        /// </summary>
        /// <param name="nonExpertResults">PerImage statistic csv file for non-experts</param>
        /// <param name="expertResults">PerImage statistic csv file for Experts</param>
        /// <param name="outputFile"></param>
        public void GetSameNonExpertAnnotationsForExpertAnnotations(string nonExpertResults, string expertResults, string outputFile)
        {
            var expertAnnotatedUrls = File.ReadAllLines(expertResults).Skip(1).Select(line => line.Split(',')[0]);

            var joinedUrls = File.ReadAllLines(nonExpertResults).Skip(1)
                .Join(expertAnnotatedUrls, line => line.Split(',')[0], item => item, (line, url) => line);

            File.WriteAllLines(outputFile, File.ReadAllLines(nonExpertResults).Take(1).Concat(joinedUrls));
        }

        /// <summary>
        /// Calculate majority voting for each image
        /// </summary>
        /// <param name="inputFile">PerImage statistic file</param>
        /// <param name="outputFile">Majority Voting output file path</param>
        public void CalculateMajorityVoting(string inputFile, string outputFile)
        {
            var lines = File.ReadAllLines(inputFile);

            var output = lines.Skip(1).Select(line => new { Votes = new Votes(line.Split(',')), Data = line })
                .Select(item => String.Format("{0},{1},{2}", item.Data, item.Votes.MajorityQ1(), item.Votes.MajorityQ2()));

            var header = lines.Take(1).Select(line => line + "Q1_Majority,Q2_Majority");

            File.WriteAllLines(outputFile, header.Concat(output));
        }

        /// <summary>
        /// Merge the majority voting results of experts and non experts
        /// </summary>
        /// <param name="expertFile">Majority voting file for experts</param>
        /// <param name="nonExpertFile">Majority voting file for non-experts</param>
        /// <param name="outputFile"></param>
        public void MergeMajorityVotings(string expertFile, string nonExpertFile, string outputFile)
        {
            var expertVotes = File.ReadAllLines(expertFile).Skip(1).Select(line => line.Split(','))
                .Select(fields => new { Url = fields[0], Q1Majority = fields[10], Q2Majority = fields[11] });

            var nonExpertVotes = File.ReadAllLines(nonExpertFile).Skip(1).Select(line => line.Split(','))
                .Select(fields => new { Url = fields[0], Q1Majority = fields[10], Q2Majority = fields[11] });

            var allVotes = expertVotes.Join(nonExpertVotes, e => e.Url, ne => ne.Url, (e, ne) =>
                String.Format("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12}",
                    e.Url,
                    e.Q1Majority,
                    ne.Q1Majority,
                    Convert.ToInt32(e.Q1Majority == "Yes") + Convert.ToInt32(ne.Q1Majority == "Yes"),
                    Convert.ToInt32(e.Q1Majority == "No") + Convert.ToInt32(ne.Q1Majority == "No"),
                    Convert.ToInt32(e.Q1Majority == "Notsure") + Convert.ToInt32(ne.Q1Majority == "Notsure"),
                    Convert.ToInt32(e.Q1Majority == "NoAgreement") + Convert.ToInt32(ne.Q1Majority == "NoAgreement"),

                    e.Q2Majority,
                    ne.Q2Majority,
                    Convert.ToInt32(e.Q2Majority == "Yes") + Convert.ToInt32(ne.Q2Majority == "Yes"),
                    Convert.ToInt32(e.Q2Majority == "No") + Convert.ToInt32(ne.Q2Majority == "No"),
                    Convert.ToInt32(e.Q2Majority == "Notsure") + Convert.ToInt32(ne.Q2Majority == "Notsure"),
                    Convert.ToInt32(e.Q2Majority == "NoAgreement") + Convert.ToInt32(ne.Q2Majority == "NoAgreement")
                )
            );

            var header = new List<string> { 
                "Url,Q1ExpertVote,Q1NonExpertVote,Q1CountYes,Q1CountNo,Q1CountNotsure,Q1CountNoAgreement," +
                "Q2ExpertVote,Q2NonExpertVote,Q2CountYes,Q2CountNo,Q2CountNotsure,Q2CountNoAgreement"
            };

            File.WriteAllLines(outputFile, header.Concat(allVotes));
        }

        /// <summary>
        /// Get some basic statics from original AMT result file
        /// </summary>
        /// <param name="inputFile">Original AMT result file in bsv (bar separated value) format</param>
        /// <param name="outputFile">Output statistics file in tsv (tab separated value) format</param>
        public void GetDataStatistic(string inputFile, string outputFile)
        {
            var output = File.ReadAllLines(inputFile).ToCsvDictionary('|')
                .Select(i => new { WorkerId = i["WorkerId"], Duration = int.Parse(i["WorkTimeInSeconds"]) })
                .GroupBy(i => i.WorkerId)
                .Select(g => new { WorkerId = g.Key, Count = g.Count(), Avg = g.Average(i => i.Duration) })
                .OrderByDescending(i => i.Count)
                .Select(i => String.Format("{0}\t{1}\t{2}", i.WorkerId, i.Count, i.Avg));

            File.WriteAllLines(outputFile, output);
        }

        /// <summary>
        /// Add photoID from database file to AMT result file
        /// </summary>
        /// <param name="photoInfoFile">Path to photo info file</param>
        /// <param name="amtResult">AMT PerImage result file</param>
        /// <param name="outputFile">OutputFile</param>
        public void MergePhotoInfo(string photoInfoFile, string amtResult, string outputFile)
        {
            var photoInfos = File.ReadAllLines(photoInfoFile).ToCsvDictionary('|')
                .Select(i => new { Id = i["PHOTO_ID"], Url = i["MEDIUM_URL"] });

            var output = File.ReadAllLines(amtResult).Skip(1)
                .Select(line => String.Format("{0},{1}",
                    photoInfos.Where(pi => line.Contains(pi.Url)).Select(pi => pi.Id).FirstOrDefault(),
                    line));

            var header = File.ReadAllLines(amtResult).Take(1).Select(line => "PHOTO_ID," + line);

            File.WriteAllLines(outputFile, header.Concat(output));
        }

        public void ReplaceWorkerIdWithIndex(string amtResultFile, string outputFile)
        {
            var header = File.ReadAllLines(amtResultFile).Take(1).ToList();
            var workerIndex = new Dictionary<String, int>();
            int index = 1;

            var output = new CsvReader(new StreamReader(amtResultFile), true).ToCsvDictionary()
                .Select(i =>
                    {
                        if (!workerIndex.ContainsKey(i["WorkerId"]))
                            workerIndex[i["WorkerId"]] = index++;

                        return header.Single().Replace("\"", "").Split(',')
                            .Select(f => f == "WorkerId" ? workerIndex[i["WorkerId"]].ToString() : i[f].Replace("\"", "\"\""))
                            .Aggregate((cur, next) => cur + ",\"" + next + "\"");
                    }).ToList();
            
            header = header.Select(h => h.Replace("WorkerId", "WorkerIndex")).ToList();
            File.WriteAllLines(outputFile, header.Concat(output));
        }

        class Votes
        {
            public int Q1Yes { get; set; }
            public int Q1No { get; set; }
            public int Q1Notsure { get; set; }

            public int Q2Yes { get; set; }
            public int Q2No { get; set; }
            public int Q2Notsure { get; set; }

            public Votes(string[] input)
            {
                Q1Yes = int.Parse(input[2]);
                Q1No = int.Parse(input[3]);
                Q1Notsure = int.Parse(input[4]);

                Q2Yes = int.Parse(input[5]);
                Q2No = int.Parse(input[6]);
                Q2Notsure = int.Parse(input[7]);
            }

            public string MajorityQ1()
            {
                if (Q1Yes >= 2)
                    return "Yes";
                else if (Q1No >= 2)
                    return "No";
                else if (Q1Notsure >= 2)
                    return "Notsure";
                else
                    return "NoAgreement";
            }

            public string MajorityQ2()
            {
                if (Q2Yes >= 2)
                    return "Yes";
                else if (Q2No >= 2)
                    return "No";
                else if (Q2Notsure >= 2)
                    return "Notsure";
                else
                    return "NoAgreement";
            }
        }


        class ImageInfo
        {
            public string HitId { get; set; }
            public string Url { get; set; }
            public string Related { get; set; }
            public string Specialty { get; set; }
            public string Category { get; set; }
            public string Familiarity { get; set; }
            public bool Approved { get; set; }

            public ImageInfo()
            {

            }

            public ImageInfo(Assignment a, int imageNo)
            {
                Category = a.Category;
                Familiarity = a.Familiarity;
                HitId = a.HitId;
                Approved = a.Approved;

                switch (imageNo)
                {
                    case 1:
                        Url = a.Url1;
                        Related = a.Related1;
                        Specialty = a.Specialty1;
                        break;
                    case 2:
                        Url = a.Url2;
                        Related = a.Related2;
                        Specialty = a.Specialty2;
                        break;
                    case 3:
                        Url = a.Url3;
                        Related = a.Related3;
                        Specialty = a.Specialty3;
                        break;
                    case 4:
                        Url = a.Url4;
                        Related = a.Related4;
                        Specialty = a.Specialty4;
                        break;
                }
            }

            public string ToString()
            {
                return String.Format("{0},{1},{2},{3},{4}", Url, Category, Related, Specialty, Familiarity);
            }
        }

        class Assignment
        {
            public int UrlCount { get; set; }
            public string HitId { get; set; }

            public string Url1 { get; set; }
            public string Url2 { get; set; }
            public string Url3 { get; set; }
            public string Url4 { get; set; }

            public string Related1 { get; set; }
            public string Related2 { get; set; }
            public string Related3 { get; set; }
            public string Related4 { get; set; }

            public string Specialty1 { get; set; }
            public string Specialty2 { get; set; }
            public string Specialty3 { get; set; }
            public string Specialty4 { get; set; }

            public string Category { get; set; }
            public string Familiarity { get; set; }
            public bool Approved { get; set; }

            private Dictionary<string, int> _fieldIndices;

            public Assignment(string[] input, Dictionary<string, int> fieldIndices)
            {
                _fieldIndices = fieldIndices;

                UrlCount = Convert.ToInt32(input[GetFiledIndex("Input.UrlCount")]);
                HitId = input[GetFiledIndex("HITId")];
                //Approved = input[GetFiledIndex("Approve")] == "x" ? true : false;

                Url1 = input[GetFiledIndex("Input.Url1")];
                Url2 = input[GetFiledIndex("Input.Url2")];
                Url3 = input[GetFiledIndex("Input.Url3")];
                Url4 = input[GetFiledIndex("Input.Url4")];

                Related1 = input[GetFiledIndex("Answer.related1")];
                Related2 = input[GetFiledIndex("Answer.related2")];
                Related3 = input[GetFiledIndex("Answer.related3")];
                Related4 = input[GetFiledIndex("Answer.related4")];

                Specialty1 = input[GetFiledIndex("Answer.specialty1")];
                Specialty2 = input[GetFiledIndex("Answer.specialty2")];
                Specialty3 = input[GetFiledIndex("Answer.specialty3")];
                Specialty4 = input[GetFiledIndex("Answer.specialty4")];

                Familiarity = input[GetFiledIndex("Answer.familiarity")];

                Category = input[GetFiledIndex("Input.Category")];
            }

            private int GetFiledIndex(string name)
            {
                return _fieldIndices.Where(f => f.Key == name).Select(f => f.Value).FirstOrDefault();
            }
        }

    }
}
